# encoding: utf-8


"""

@author: tongzhenguo

@time: 2021/5/2 上午11:56

@desc:

证券宝文档:http://baostock.com/baostock/index.php/%E9%A6%96%E9%A1%B5
tushare文档:https://waditu.com/document/2
akshare文档:https://www.akshare.xyz/zh_CN/latest/introduction.html

存放股票的基本数据

"""
import os
import sys
import traceback
from time import sleep

import baostock as bs
import demjson
import pandas as pd
import requests
import tushare as ts
from bs4 import BeautifulSoup
from numpy import double
from pandas import DataFrame
from tqdm import tqdm

from const import ALL_STOCK_A_FILE, TRADE_CAL_FILE
from date_util import now_year, month_to_quarterly, last_year, now_date_fmt_8, last_n_year, quarterly_report_date, \
    get_month_last_trade_date
from mongo_db import MongoDB

mongo_db = MongoDB(uri="mongodb://127.0.0.1:27017", database="stock")


class StockBase(object):
    def __init__(self):
        self.date_s = now_date_fmt_8()
        self.sep = ","
        self.lg = bs.login(user_id="anonymous", password="123456")
        # NOTE 这里记得替换成自己的token奥！
        self.tushare_token = "357f92fd3836f2d018d20b9b840897abb3e5c9a62e17895b413e05fe"
        # ts.set_token(self.tushare_token)
        self.tushare_pro_api = ts.pro_api(self.tushare_token)
        self.lrb_columns = "序号,股票代码,股票简称,净利润,净利润同比,营业总收入,营业总收入同比,营业总支出-营业支出,营业总支出-销售费用," \
                           "营业总支出-管理费用,营业总支出-财务费用,营业总支出-营业总支出,营业利润,利润总额,公告日期".split(",")
        self.zcfzb_columns = "序号,股票代码,股票简称,资产-货币资金,资产-应收账款,资产-存货,资产-总资产,资产-总资产同比,负债-应付账款," \
                             "负债-预收账款,负债-总负债,负债-总负债同比,资产负债率,股东权益合计,公告日期".split(",")
        self.xjllb_columns = "序号,股票代码,股票简称,净现金流-净现金流,净现金流-同比增长,经营性现金流-现金流量净额,经营性现金流-净现金流占比," \
                             "投资性现金流-现金流量净额,投资性现金流-净现金流占比,融资性现金流-现金流量净额,融资性现金流-净现金流占比," \
                             "公告日期".split(",")
        self.financial_indicator_columns = [
            '摊薄每股收益(元)', '加权每股收益(元)', '每股收益_调整后(元)', '扣除非经常性损益后的每股收益(元)',
            '每股净资产_调整前(元)', '每股净资产_调整后(元)', '每股经营性现金流(元)', '每股资本公积金(元)',
            '每股未分配利润(元)', '调整后的每股净资产(元)', '总资产利润率(%)', '主营业务利润率(%)', '总资产净利润率(%)',
            '成本费用利润率(%)', '营业利润率(%)', '主营业务成本率(%)', '销售净利率(%)', '股本报酬率(%)',
            '净资产报酬率(%)', '资产报酬率(%)', '销售毛利率(%)', '三项费用比重', '非主营比重', '主营利润比重',
            '股息发放率(%)', '投资收益率(%)', '主营业务利润(元)', '净资产收益率(%)', '加权净资产收益率(%)',
            '扣除非经常性损益后的净利润(元)', '主营业务收入增长率(%)', '净利润增长率(%)', '净资产增长率(%)',
            '总资产增长率(%)', '应收账款周转率(次)', '应收账款周转天数(天)', '存货周转天数(天)', '存货周转率(次)',
            '固定资产周转率(次)', '总资产周转率(次)', '总资产周转天数(天)', '流动资产周转率(次)', '流动资产周转天数(天)',
            '股东权益周转率(次)', '流动比率', '速动比率', '现金比率(%)', '利息支付倍数', '长期债务与营运资金比率(%)',
            '股东权益比率(%)', '长期负债比率(%)', '股东权益与固定资产比率(%)', '负债与所有者权益比率(%)',
            '长期资产与长期资金比率(%)', '资本化比率(%)', '固定资产净值率(%)', '资本固定化比率(%)', '产权比率(%)',
            '清算价值比率(%)', '固定资产比重(%)', '资产负债率(%)', '总资产(元)', '经营现金净流量对销售收入比率(%)',
            '资产的经营现金流量回报率(%)', '经营现金净流量与净利润的比率(%)', '经营现金净流量对负债比率(%)', '现金流量比率(%)',
            '短期股票投资(元)', '短期债券投资(元)', '短期其它经营性投资(元)', '长期股票投资(元)', '长期债券投资(元)',
            '长期其它经营性投资(元)', '1年以内应收帐款(元)', '1-2年以内应收帐款(元)', '2-3年以内应收帐款(元)',
            '3年以内应收帐款(元)', '1年以内预付货款(元)', '1-2年以内预付货款(元)', '2-3年以内预付货款(元)',
            '3年以内预付货款(元)', '1年以内其它应收款(元)', '1-2年以内其它应收款(元)', '2-3年以内其它应收款(元)',
            '3年以内其它应收款(元)']
        self.goodwill_columns = "股票代码,商誉,商誉占净资产比例,净利率,上年商誉".split(",")
        self.basic_columns = "code,地域,所属行业,上市日期,市场类型（主板/创业板/科创板/CDR)".split(",")
        self.frequency_dict = {"d": "日", "w": "周", "m": "月"}
        df = self.tushare_pro_api.trade_cal(exchange='', start_date=f"{now_year()}0101", end_date=f"{now_year()}1231")
        df.to_csv(TRADE_CAL_FILE, index=False)
        self.trade_cal_df = pd.read_csv(TRADE_CAL_FILE, dtype={"cal_date": str, "is_open": str})

    @staticmethod
    def close():
        bs.logout()

    def trade_cal(self, **kwargs):
        """获取交易日历"""
        start_date = kwargs["start_date"] if "start_date" in kwargs else f"{now_year()}0101"
        end_data = kwargs["end_date"] if "end_date" in kwargs else f"{now_year()}1231"
        df = self.tushare_pro_api.trade_cal(exchange='', start_date=start_date, end_date=end_data)
        df.to_csv(TRADE_CAL_FILE, index=False)

    def download_all_stock(self):
        """获取A股上市公司列表"""
        # 查询当前所有正常上市交易的股票列表
        # https://waditu.com/document/2?doc_id=25
        data = self.tushare_pro_api.query('stock_basic', exchange='', list_status='L',
                                          fields='ts_code,symbol,name,area,industry,list_date,market')
        data = data.rename(
            columns={"ts_code": "TS代码", "symbol": "code", "name": "name", "area": "地域", "industry": "所属行业",
                     "list_date": "上市日期", "market": "市场类型（主板/创业板/科创板/CDR)"})
        # 避免股票代码000001变成自动推断为数字1
        data["code"] = data["code"].apply(str)
        data.to_csv(ALL_STOCK_A_FILE, index=False)

    def stock_basic(self):
        """获取A股上市公司列表"""
        # 查询当前所有正常上市交易的股票列表
        # https://waditu.com/document/2?doc_id=25
        data = self.tushare_pro_api.query('stock_basic', exchange='', list_status='L',
                                          fields='ts_code,symbol,name,area,industry,list_date,market')
        data = data.rename(
            columns={"ts_code": "TS代码", "symbol": "code", "name": "name", "area": "地域", "industry": "所属行业",
                     "list_date": "上市日期", "market": "市场类型（主板/创业板/科创板/CDR)"})
        # 避免股票代码000001变成自动推断为数字1
        data["code"] = data["code"].apply(str)
        if data.shape[0] == 0:
            print("error stock_basic get fail !!!")
            pass
        for i in range(data.shape[0]):
            code = data["code"].values[i]
            feature_dict = dict(zip(self.basic_columns, data[self.basic_columns].values[i]))
            # 确保上市时间为数值型
            feature_dict["上市日期"] = int(feature_dict["上市日期"])
            mongo_db.upsert("stock", {"_id": code}, feature_dict)

    @staticmethod
    def get_history_k_data(code, start_date, end_date, frequency="d"):
        """获取历史k线数据"""
        query_code = "sh.%s" % code if int(code) >= 600000 else "sz.%s" % code
        rs = bs.query_history_k_data_plus(query_code, "date,open,high,low,close,volume,turn", start_date, end_date,
                                          frequency=frequency,
                                          adjustflag="3")
        # 输出结果
        data_list = []
        while (rs.error_code == '0') & rs.next():
            # 获取一条记录，将记录合并在一起
            data_list.append(rs.get_row_data())
        result = pd.DataFrame(data_list, columns=rs.fields)
        for col in ["open", "high", "low", "close", "volume", "turn"]:
            result[col] = result[col].apply(float)
        return result

    def update_st_data(self, stock_all_df, date):
        """更新是否是退市警告股票
        *ST---公司经营连续三年亏损，退市预警。
        ST----公司经营连续二年亏损，特别处理。
        S*ST--公司经营连续三年亏损，退市预警+还没有完成股改。
        SST---公司经营连续二年亏损，特别处理+还没有完成股改。
        S----还没有完成股改。
        """
        for code in stock_all_df['code'].values:
            print(f"starting write {code} st data !!!")
            if code[0] not in ["0", "6"] or code[:3] == "688":
                # 科创板买不了，先不看！
                continue
            query_code = "sh.%s" % code if int(code) >= 600000 else "sz.%s" % code
            _id = f"{code}-退市警告股票-{now_year()}"
            if mongo_db.exists_one("collect_task_info", {"_id": _id}):
                print("task %s done" % _id)
                continue
            try:
                rs = bs.query_history_k_data_plus(query_code, "isST",
                                                  start_date=date, end_date=date,
                                                  frequency="d", adjustflag="3")
                result_list = []
                while (rs.error_code == '0') & rs.next():
                    result_list.append(rs.get_row_data())
                result = pd.DataFrame(result_list, columns=rs.fields)
                if result.empty:
                    continue
                feature_dict = {"isST": result["isST"].values[-1]}
                if len(feature_dict) == 0:
                    print(f"error update_st_data {code} feature_dict empty!!!")
                else:
                    mongo_db.upsert("stock", {"_id": code}, feature_dict)
                    mongo_db.add("collect_task_info", {"_id": _id, "done": 1})
            except Exception as e:
                traceback.print_exc()
                print("error update_st_data msg:%s" % repr(e))

    def write_history_k_data(self, stock_all_df, start_date, end_date, frequency="d", adjustflag="3", **kwargs):
        """历史k线数据入库"""
        frequency_type = self.frequency_dict[frequency]
        for code in stock_all_df['code'].values:
            print(f"starting write {code} k data !!!")
            # # 当前交易日17:30，完成日K线数据入库；
            # if int(now_date_fmt(fmt="HH")) <= 16:
            #     return
            if code[0] not in ["0", "6"] or code[:3] == "688":
                # 科创板买不了，先不看！
                continue
            query_code = "sh.%s" % code if int(code) >= 600000 else "sz.%s" % code
            _id = f"{code}-历史{frequency_type}k线数据-{start_date}-{end_date}"
            if mongo_db.exists_one("collect_task_info", {"_id": _id}):
                print("task %s done" % _id)
                continue
            try:
                start_date_10 = start_date[:4] + "-" + start_date[4:6] + "-" + start_date[6:]
                if not end_date:
                    end_date_10 = None
                else:
                    end_date_10 = end_date[:4] + "-" + end_date[4:6] + "-" + end_date[6:]
                rs = bs.query_history_k_data_plus(query_code, "date,open,high,low,close",
                                                  start_date=start_date_10, end_date=end_date_10,
                                                  frequency=frequency, adjustflag=adjustflag)
                result_list = []
                while (rs.error_code == '0') & rs.next():
                    result_list.append(rs.get_row_data())
                result = pd.DataFrame(result_list, columns=rs.fields)
                feature_dict = dict()
                for col, name in zip(["open", "high", "low", "close"], ["开盘价", "最高价", "最低价", "收盘价"]):
                    col_list = [f"{d.replace('-', '')}{frequency_type}-{name}" for d in result["date"].tolist()]
                    val_list = [c for c in result[col].to_list()]
                    feature_dict.update(dict(zip(col_list, val_list)))
                if len(feature_dict) == 0:
                    print(f"error write_history_k_data {code} feature_dict empty!!!")
                else:
                    # 通过日K同时更新下月K
                    if "sync_update_month_k" in kwargs and kwargs["sync_update_month_k"]:
                        month_last_trade_date = get_month_last_trade_date(start_date[:4], start_date[4:6],
                                                                          self.trade_cal_df)
                    feature_dict[f"{month_last_trade_date}月-收盘价"] = feature_dict[f"{start_date}日-收盘价"]
                    mongo_db.upsert("stock", {"_id": code}, feature_dict)
                    mongo_db.add("collect_task_info", {"_id": _id, "done": 1})
            except Exception as e:
                traceback.print_exc()
                raise RuntimeError("error write_history_k_data msg:%s" % repr(e))

    def get_profit_statement_v2(self, date, stock_all_df):
        """利润表
        目标地址:http://data.eastmoney.com/bbsj/202003/lrb.html
        """

        def stock_em_lrb(date: str = "20200331") -> pd.DataFrame:
            """
            东方财富-数据中心-年报季报-业绩快报-利润表
            http://data.eastmoney.com/bbsj/202003/lrb.html
            :param date: "20200331", "20200630", "20200930", "20201231"; 从 20100331 开始
            :type date: str
            :return: 利润表
            :rtype: pandas.DataFrame
            """
            url = "http://datacenter.eastmoney.com/api/data/get"
            params = {
                "st": "NOTICE_DATE,SECURITY_CODE",
                "sr": "-1,-1",
                "ps": "5000",
                "p": "1",
                "type": "RPT_DMSK_FN_INCOME",
                "sty": "ALL",
                "token": "894050c76af8597a853f5b408b759f5d",
                "filter": f"(REPORT_DATE='{'-'.join([date[:4], date[4:6], date[6:]])}')",
            }
            r = requests.get(url, params=params)
            data_json = r.json()
            if not data_json or "result" not in data_json or not data_json["result"] or "pages" not in data_json[
                "result"]:
                return None
            page_num = data_json["result"]["pages"]
            big_df = pd.DataFrame()
            for page in tqdm(range(1, page_num + 1)):
                params = {
                    "st": "NOTICE_DATE,SECURITY_CODE",
                    "sr": "-1,-1",
                    "ps": "5000",
                    "p": page,
                    "type": "RPT_DMSK_FN_INCOME",
                    "sty": "ALL",
                    "token": "894050c76af8597a853f5b408b759f5d",
                    "filter": f"(REPORT_DATE='{'-'.join([date[:4], date[4:6], date[6:]])}')",
                }
                r = requests.get(url, params=params)
                data_json = r.json()
                temp_df = pd.DataFrame(data_json["result"]["data"])
                big_df = big_df.append(temp_df, ignore_index=True)

            big_df.reset_index(inplace=True)
            big_df["index"] = range(1, len(big_df) + 1)
            big_df.columns = [
                "序号",
                "_",
                "股票代码",
                "_",
                "_",
                "股票简称",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "公告日期",
                "_",
                "净利润",
                "营业总收入",
                "营业总支出-营业总支出",
                "_",
                "营业总支出-营业支出",
                "_",
                "_",
                "营业总支出-销售费用",
                "营业总支出-管理费用",
                "营业总支出-财务费用",
                "营业利润",
                "利润总额",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "营业总收入同比",
                "_",
                "净利润同比",
                "_",
                "_",
            ]
            big_df = big_df[
                [
                    "序号",
                    "股票代码",
                    "股票简称",
                    "净利润",
                    "净利润同比",
                    "营业总收入",
                    "营业总收入同比",
                    "营业总支出-营业支出",
                    "营业总支出-销售费用",
                    "营业总支出-管理费用",
                    "营业总支出-财务费用",
                    "营业总支出-营业总支出",
                    "营业利润",
                    "利润总额",
                    "公告日期",
                ]
            ]
            return big_df

        y = date[:4]
        stock_em_lrb_df = stock_em_lrb(date)
        if not isinstance(stock_em_lrb_df, DataFrame):
            print("error stock_em_lrb return data empty !!!")
            return
        lrb_columns = self.lrb_columns.copy()
        lrb_columns.remove("公告日期")
        lrb_columns.remove("序号")
        lrb_columns.remove("股票代码")
        for code in stock_all_df['code'].values:
            _id = "%s-%s-%s" % (code, "年度利润表", date)
            if mongo_db.exists_one("collect_task_info", {"_id": _id}):
                print("task %s done" % _id)
            elif code[0] not in ["0", "6"]:
                # 科创板买不了，先不看！
                continue
            else:
                try:
                    df = stock_em_lrb_df[stock_em_lrb_df['股票代码'] == code]
                    columns_dict = {col: "%s年度-%s" % (y, col) for col in lrb_columns}
                    df = df.rename(columns=columns_dict)
                    # todo 有一些空值(nan)就先不处理了
                    columns = [v for k, v in columns_dict.items()]
                    if df.shape[0] == 0:
                        print("code %s %s年度利润表数据不存在" % (code, y))
                        continue
                    feature_dict = dict(zip(columns, df.head(1)[columns].values[0]))
                    mongo_db.upsert("stock", {"_id": code}, feature_dict)
                    mongo_db.add("collect_task_info", {"_id": _id, "done": 1})
                except Exception as e:
                    traceback.print_stack()
                    print("error get_profit_statement_v2 _id: %s , msg: %s " % (_id, repr(e)))
                    # raise RuntimeError("error get_profit_statement_v2 _id: %s , msg: %s " % (_id, repr(e)))

    def get_cash_flow_statement_v2(self, date, stock_all_df):
        """现金流表
        目标地址: http://data.eastmoney.com/bbsj/202003/xjll.html
        """

        def stock_em_xjll(date: str = "20200331") -> pd.DataFrame:
            """
            东方财富-数据中心-年报季报-业绩快报-现金流量表
            http://data.eastmoney.com/bbsj/202003/xjll.html
            :param date: "20200331", "20200630", "20200930", "20201231"; 从 20100331 开始
            :type date: str
            :return: 现金流量表
            :rtype: pandas.DataFrame
            """
            url = "http://datacenter.eastmoney.com/api/data/get"
            params = {
                "st": "NOTICE_DATE,SECURITY_CODE",
                "sr": "-1,-1",
                "ps": "5000",
                "p": "1",
                "type": "RPT_DMSK_FN_CASHFLOW",
                "sty": "ALL",
                "token": "894050c76af8597a853f5b408b759f5d",
                "filter": f"(REPORT_DATE='{'-'.join([date[:4], date[4:6], date[6:]])}')",
            }
            r = requests.get(url, params=params)
            data_json = r.json()
            if not data_json or "result" not in data_json or not data_json["result"] or "pages" not in data_json[
                "result"]:
                return None
            page_num = data_json["result"]["pages"]
            big_df = pd.DataFrame()
            for page in tqdm(range(1, page_num + 1)):
                params = {
                    "st": "NOTICE_DATE,SECURITY_CODE",
                    "sr": "-1,-1",
                    "ps": "5000",
                    "p": page,
                    "type": "RPT_DMSK_FN_CASHFLOW",
                    "sty": "ALL",
                    "token": "894050c76af8597a853f5b408b759f5d",
                    "filter": f"(REPORT_DATE='{'-'.join([date[:4], date[4:6], date[6:]])}')",
                }
                r = requests.get(url, params=params)
                data_json = r.json()
                temp_df = pd.DataFrame(data_json["result"]["data"])
                big_df = big_df.append(temp_df, ignore_index=True)

            big_df.reset_index(inplace=True)
            big_df["index"] = range(1, len(big_df) + 1)
            big_df.columns = [
                "序号",
                "_",
                "股票代码",
                "_",
                "_",
                "股票简称",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "公告日期",
                "_",
                "经营性现金流-现金流量净额",
                "经营性现金流-净现金流占比",
                "_",
                "_",
                "_",
                "_",
                "投资性现金流-现金流量净额",
                "投资性现金流-净现金流占比",
                "_",
                "_",
                "_",
                "_",
                "融资性现金流-现金流量净额",
                "融资性现金流-净现金流占比",
                "净现金流-净现金流",
                "净现金流-同比增长",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
            ]
            big_df = big_df[
                [
                    "序号",
                    "股票代码",
                    "股票简称",
                    "净现金流-净现金流",
                    "净现金流-同比增长",
                    "经营性现金流-现金流量净额",
                    "经营性现金流-净现金流占比",
                    "投资性现金流-现金流量净额",
                    "投资性现金流-净现金流占比",
                    "融资性现金流-现金流量净额",
                    "融资性现金流-净现金流占比",
                    "公告日期",
                ]
            ]
            return big_df

        y = date[:4]
        stock_em_xjll_df = stock_em_xjll(date)
        if not isinstance(stock_em_xjll_df, DataFrame):
            print("error stock_em_xjll return data empty !!!")
            return
        xjllb_columns = self.xjllb_columns.copy()
        xjllb_columns.remove("公告日期")
        xjllb_columns.remove("序号")
        xjllb_columns.remove("股票代码")
        for code in stock_all_df['code'].values:
            _id = "%s-%s-%s" % (code, "年度现金流量表", date)
            if mongo_db.exists_one("collect_task_info", {"_id": _id}):
                print("task %s done" % _id)
            elif code[0] not in ["0", "6"]:
                # 科创板买不了，先不看！
                continue
            else:
                try:
                    df = stock_em_xjll_df[stock_em_xjll_df['股票代码'] == code]
                    columns_dict = {col: "%s年度-%s" % (y, col) for col in xjllb_columns}
                    df = df.rename(columns=columns_dict)
                    # todo 有一些空值(nan)就先不处理了
                    columns = [v for k, v in columns_dict.items()]
                    if df.shape[0] == 0:
                        print("code %s %s年度现金流量表数据不存在" % (code, y))
                        continue
                    feature_dict = dict(zip(columns, df.head(1)[columns].values[0]))
                    mongo_db.upsert("stock", {"_id": code}, feature_dict)
                    mongo_db.add("collect_task_info", {"_id": _id, "done": 1})
                except Exception as e:
                    traceback.print_stack()
                    print("error get_cash_flow_statement_v2 _id: %s , msg: %s " % (_id, repr(e)))
                    # raise RuntimeError("error get_cash_flow_statement_v2 _id: %s , msg: %s " % (_id, repr(e)))

    def get_assets_and_liabilities_statement_v2(self, date, stock_all_df):
        """资产负债表
        目标地址: http://data.eastmoney.com/bbsj/202003/zcfz.html
        """

        def stock_em_zcfz(date: str = "20200331") -> pd.DataFrame:
            """
            东方财富-数据中心-年报季报-业绩快报-资产负债表
            http://data.eastmoney.com/bbsj/202003/zcfz.html
            :param date: "20200331", "20200630", "20200930", "20201231"; 从 20100331 开始
            :type date: str
            :return: 资产负债表
            :rtype: pandas.DataFrame
            """
            url = "http://datacenter.eastmoney.com/api/data/get"
            params = {
                "st": "NOTICE_DATE,SECURITY_CODE",
                "sr": "-1,-1",
                "ps": "5000",
                "p": "1",
                "type": "RPT_DMSK_FN_BALANCE",
                "sty": "ALL",
                "token": "894050c76af8597a853f5b408b759f5d",
                "filter": f"(REPORT_DATE='{'-'.join([date[:4], date[4:6], date[6:]])}')",
            }
            r = requests.get(url, params=params)
            data_json = r.json()
            if not data_json or "result" not in data_json or not data_json["result"] or "pages" not in data_json[
                "result"]:
                return None
            page_num = data_json["result"]["pages"]
            big_df = pd.DataFrame()
            for page in tqdm(range(1, page_num + 1)):
                params = {
                    "st": "NOTICE_DATE,SECURITY_CODE",
                    "sr": "-1,-1",
                    "ps": "5000",
                    "p": page,
                    "type": "RPT_DMSK_FN_BALANCE",
                    "sty": "ALL",
                    "token": "894050c76af8597a853f5b408b759f5d",
                    "filter": f"(REPORT_DATE='{'-'.join([date[:4], date[4:6], date[6:]])}')",
                }
                r = requests.get(url, params=params)
                data_json = r.json()
                temp_df = pd.DataFrame(data_json["result"]["data"])
                big_df = big_df.append(temp_df, ignore_index=True)

            big_df.reset_index(inplace=True)
            big_df["index"] = range(1, len(big_df) + 1)
            big_df.columns = [
                "序号",
                "_",
                "股票代码",
                "_",
                "_",
                "股票简称",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "公告日期",
                "_",
                "资产-总资产",
                "_",
                "资产-货币资金",
                "_",
                "资产-应收账款",
                "_",
                "资产-存货",
                "_",
                "负债-总负债",
                "负债-应付账款",
                "_",
                "负债-预收账款",
                "_",
                "股东权益合计",
                "_",
                "资产-总资产同比",
                "负债-总负债同比",
                "_",
                "资产负债率",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
                "_",
            ]
            big_df = big_df[
                [
                    "序号",
                    "股票代码",
                    "股票简称",
                    "资产-货币资金",
                    "资产-应收账款",
                    "资产-存货",
                    "资产-总资产",
                    "资产-总资产同比",
                    "负债-应付账款",
                    "负债-预收账款",
                    "负债-总负债",
                    "负债-总负债同比",
                    "资产负债率",
                    "股东权益合计",
                    "公告日期",
                ]
            ]
            return big_df

        y = date[:4]
        stock_em_zcfz_df = stock_em_zcfz(date)
        if not isinstance(stock_em_zcfz_df, DataFrame):
            print("error stock_em_zcfz return data empty !!!")
            return
        zcfzb_columns = self.zcfzb_columns.copy()
        zcfzb_columns.remove("公告日期")
        zcfzb_columns.remove("序号")
        zcfzb_columns.remove("股票代码")
        for code in stock_all_df['code'].values:
            _id = "%s-%s-%s" % (code, "年度资产负债表", date)
            if mongo_db.exists_one("collect_task_info", {"_id": _id}):
                print("task %s done" % _id)
            elif code[0] not in ["0", "6"]:
                # 科创板买不了，先不看！
                continue
            else:
                try:
                    df = stock_em_zcfz_df[stock_em_zcfz_df['股票代码'] == code]
                    columns_dict = {col: "%s年度-%s" % (y, col) for col in zcfzb_columns}
                    df = df.rename(columns=columns_dict)
                    # todo 有一些空值(nan)就先不处理了
                    columns = [v for k, v in columns_dict.items()]
                    if df.shape[0] == 0:
                        print("code %s %s年度现金流量表数据不存在" % (code, y))
                        continue
                    feature_dict = dict(zip(columns, df.head(1)[columns].values[0]))
                    mongo_db.upsert("stock", {"_id": code}, feature_dict)
                    mongo_db.add("collect_task_info", {"_id": _id, "done": 1})
                except Exception as e:
                    traceback.print_stack()
                    raise RuntimeError(
                        "error get_assets_and_liabilities_statement_v2 _id: %s , msg: %s " % (_id, repr(e)))

    def get_goodwill_data_v2(self, date):
        """获取商誉数据"""

        def _get_page_num_sy_list(symbol: str = "沪市主板", trade_date: str = "2019-12-31") -> int:
            """
            东方财富网-数据中心-特色数据-商誉-个股商誉明细
            http://data.eastmoney.com/sy/list.html
            :param symbol: choice of {"沪市主板", "深市主板", "中小板", "创业板", "沪深两市"}
            :type symbol: str
            :param trade_date: 参考网站指定的数据日期
            :type trade_date: str
            :return: 个股商誉明细 的总页数
            :rtype: int
            """
            symbol_dict = {
                "沪市主板": f"""(TRADE_BOARD="shzb")(REPORT_DATE='{trade_date}')""",
                "深市主板": f"""(TRADE_BOARD="szzb")(REPORT_DATE='{trade_date}')""",
                "中小板": f"""(TRADE_BOARD="zxb")(REPORT_DATE='{trade_date}')""",
                "创业板": f"""(TRADE_BOARD="cyb")(REPORT_DATE='{trade_date}')""",
                "沪深两市": f"(REPORT_DATE='{trade_date}')",
            }
            url = "http://datacenter.eastmoney.com/api/data/get"
            params = {
                "type": "RPT_GOODWILL_STOCKDETAILS",
                "sty": "ALL",
                "p": "1",
                "ps": "50",
                "sr": "-1,-1",
                "st": "NOTICE_DATE,SECURITY_CODE",
                "var": "QvxsKBaH",
                "filter": symbol_dict[symbol],
                "rt": "53324381",
            }
            res = requests.get(url, params=params)
            data_json = demjson.decode(res.text[res.text.find("{"):-1])
            return data_json["result"]["pages"]

        def stock_em_sy_list(symbol: str = "深市主板", trade_date: str = "2019-12-31") -> pd.DataFrame:
            """
            东方财富网-数据中心-特色数据-商誉-个股商誉明细
            http://data.eastmoney.com/sy/list.html
            :param symbol: choice of {"沪市主板", "深市主板", "中小板", "创业板", "沪深两市"}
            :type symbol: str
            :param trade_date: 参考网站指定的数据日期
            :type trade_date: str
            :return: 个股商誉明细数据
            :rtype: pandas.DataFrame
            """
            symbol_dict = {
                "沪市主板": f"""(TRADE_BOARD="shzb")(REPORT_DATE='{trade_date}')""",
                "深市主板": f"""(TRADE_BOARD="szzb")(REPORT_DATE='{trade_date}')""",
                "中小板": f"""(TRADE_BOARD="zxb")(REPORT_DATE='{trade_date}')""",
                "创业板": f"""(TRADE_BOARD="cyb")(REPORT_DATE='{trade_date}')""",
                "沪深两市": f"(REPORT_DATE='{trade_date}')",
            }
            url = "http://datacenter.eastmoney.com/api/data/get"
            page_num = _get_page_num_sy_list(symbol=symbol, trade_date=trade_date)
            temp_df = pd.DataFrame()
            for page in tqdm(range(1, page_num + 1)):
                params = {
                    "type": "RPT_GOODWILL_STOCKDETAILS",
                    "sty": "ALL",
                    "p": str(page),
                    "ps": "50",
                    "sr": "-1,-1",
                    "st": "NOTICE_DATE,SECURITY_CODE",
                    "var": "QvxsKBaH",
                    "filter": symbol_dict[symbol],
                    "rt": "53324381",
                }
                res = requests.get(url, params=params)
                data_text = res.text
                # 处理不标准json
                data_json = demjson.decode(data_text[data_text.find("{"): -1])
                temp_df = temp_df.append(pd.DataFrame(data_json["result"]["data"]), ignore_index=True)
            temp_df.columns = [
                "_",
                "股票代码",
                "股票简称",
                "_",
                "_",
                "_",
                "_",
                "商誉",
                "_",
                "_",
                "_",
                "公告日期",
                "报告期",
                "上年商誉",
                "_",
                "_",
                "商誉占净资产比例",
                "_",
                "净利率",
                "_",
                "净利润同比",
                "_",
                "_",
                "_",
                "_",
            ]
            temp_df = temp_df[
                [
                    "股票代码",
                    "股票简称",
                    "商誉",
                    "商誉占净资产比例",
                    "净利率",
                    "净利润同比",
                    "上年商誉",
                    "公告日期",
                ]
            ]
            temp_df["公告日期"] = pd.to_datetime(temp_df["公告日期"])
            temp_df["股票代码"] = temp_df["股票代码"].str.zfill(6)
            return temp_df

        # stock_em_sy_list_df = ak.stock_em_sy_list(symbol="沪深两市", trade_date=date)
        stock_em_sy_list_df = stock_em_sy_list(symbol="沪深两市", trade_date=date)
        stock_em_sy_list_df = stock_em_sy_list_df[self.goodwill_columns]
        columns = self.goodwill_columns.copy()
        columns.remove("股票代码")
        if stock_em_sy_list_df.shape[0] == 0:
            pass
        for i in range(stock_em_sy_list_df.shape[0]):
            code = stock_em_sy_list_df["股票代码"].values[i]
            feature_dict = dict(zip(columns, stock_em_sy_list_df[columns].values[i]))
            mongo_db.upsert("stock", {"_id": code}, feature_dict)

    @staticmethod
    def get_indicator_data_v2(start_date, end_date, frequency, adjustflag, mongo_db, stock_all_df, **kwargs):
        """获取估值指标"""

        def stock_a_lg_indicator(stock):
            url = f"https://www.legulegu.com/s/base-info/{stock}"
            r = requests.get(url)
            temp_json = r.json()
            if not temp_json or len(temp_json) == 0 or not temp_json["data"]:
                return None
            temp_df = pd.DataFrame(temp_json["data"]["items"], columns=temp_json["data"]["fields"])
            temp_df["trade_date"] = pd.to_datetime(temp_df["trade_date"])
            temp_df.iloc[:, 1:] = temp_df.iloc[:, 1:].astype(float)
            return temp_df

        for code in stock_all_df['code'].values:
            print("code:%s 获取总市值、市盈率(滚动)、市净率、市销率(滚动)、市现率(滚动)、股息率" % code)
            # # 当前交易日17:30，完成日K线数据入库；
            # if int(now_date_fmt(fmt="HH")) <= 16:
            #     return
            query_code = "sh.%s" % code if int(code) >= 600000 else "sz.%s" % code
            trade_cal_df = pd.read_csv(TRADE_CAL_FILE, dtype={"cal_date": str, "is_open": str})
            _id = "%s-%s-%s" % (code, "估值指标", start_date)
            if trade_cal_df[trade_cal_df.cal_date == end_date]["is_open"].values[0] == "0":
                # 非交易日跳过
                print("date %s is not open " % end_date)
            elif mongo_db.exists_one("collect_task_info", {"_id": _id}):
                print("task %s done" % _id)
            elif code[0] == "3" or code[:3] == "688":
                # 科创板买不了，先不看！
                continue
            else:
                try:
                    start_date_10 = start_date[:4] + "-" + start_date[4:6] + "-" + start_date[
                                                                                   6:] if start_date else None
                    date_10 = end_date[:4] + "-" + end_date[4:6] + "-" + end_date[6:] if end_date else None
                    rs = bs.query_history_k_data_plus(query_code,
                                                      "date,code,close,peTTM,pbMRQ,psTTM,pcfNcfTTM",
                                                      start_date=start_date_10, end_date=date_10,
                                                      frequency=frequency, adjustflag=adjustflag)
                    result_list = []
                    while (rs.error_code == '0') & rs.next():
                        result_list.append(rs.get_row_data())
                    result = pd.DataFrame(result_list, columns=rs.fields)
                    output_columns = ["peTTM", "pbMRQ", "psTTM", "pcfNcfTTM", "close"]
                    output_names = ["滚动市盈率", "市净率", "滚动市销率", "滚动市现率", "收盘价"]
                    # result = result.rename(
                    #     columns={"peTTM": end_date + "日-滚动市盈率", "pbMRQ": end_date + "日-市净率",
                    #              "psTTM": end_date + "日-滚动市销率",
                    #              "pcfNcfTTM": end_date + "日-滚动市现率", "close": end_date + "日-收盘价"})
                    result["code"] = code
                    # output_columns = [f"{end_date}日-滚动市盈率", f"{end_date}日-市净率", f"{end_date}日-滚动市销率",
                    #                   f"{end_date}日-滚动市现率", f"{end_date}日-收盘价"]
                    if "stock_a_lg_indicator" not in kwargs or kwargs["stock_a_lg_indicator"]:
                        # 股息率,总市值
                        # 目标地址: https://www.legulegu.com/stocklist
                        # stock_a_indicator_df = ak.stock_a_lg_indicator(stock=code)
                        stock_a_indicator_df = stock_a_lg_indicator(stock=code)
                        if isinstance(stock_a_indicator_df, DataFrame) and stock_a_indicator_df.shape[0] == 0:
                            continue
                        elif type(stock_a_indicator_df) is None:
                            continue
                        stock_a_indicator_df = stock_a_indicator_df[stock_a_indicator_df.trade_date == end_date]
                        stock_a_indicator_df = stock_a_indicator_df[["dv_ttm", "total_mv"]]
                        # stock_a_indicator_df = stock_a_indicator_df.rename(
                        #     columns={"dv_ttm": end_date + "日-滚动股息率", "total_mv": end_date + "日-总市值"})
                        stock_a_indicator_df["code"] = code
                        result = pd.merge(result, stock_a_indicator_df, how="left", on="code")
                        output_columns.extend(["dv_ttm", "total_mv"])
                        output_names.extend(["滚动股息率", "总市值"])
                    # result = result[output_columns]
                    feature_dict = dict()
                    for col, name in zip(output_columns, output_names):
                        col_list = [f"{d.replace('-', '')}日-{name}" for d in result["date"].tolist()]
                        val_list = [c for c in result[col].to_list()]
                        feature_dict.update(dict(zip(col_list, val_list)))
                    if len(feature_dict) == 0:
                        print(f"error get_indicator_data_v2 {code} feature_dict empty!!!")
                    else:
                        mongo_db.upsert("stock", {"_id": code}, feature_dict)
                        mongo_db.add("collect_task_info", {"_id": _id, "done": 1})
                except Exception as e:
                    traceback.print_exc()
                    print("error query_history_k_data_plus msg:%s" % repr(e))

    def stock_financial_analysis_indicator_v2(self, sink_db_date, indicator_list, y_list, mongo_db: MongoDB,
                                              stock_all_df):
        """财务指标(分季度)
        目标地址:https://money.finance.sina.com.cn/corp/go.php/vFD_FinancialGuideLine/stockid/600004/ctrl/2019/displaytype/4.phtml
        """

        # indicator_list = ["每股指标", "盈利能力", "成长能力", "营运能力", "偿债及资本结构", "现金流量", "其他指标"]

        def stock_financial_analysis_indicator_fn(stock: str, indicator_list) -> pd.DataFrame:
            """
            新浪财经-财务分析-财务指标
            https://money.finance.sina.com.cn/corp/go.php/vFD_FinancialGuideLine/stockid/600004/ctrl/2019/displaytype/4.phtml
            :param stock: 股票代码
            :type stock: str
            :return: 新浪财经-财务分析-财务指标
            :rtype: pandas.DataFrame
            """
            url = f"https://money.finance.sina.com.cn/corp/go.php/vFD_FinancialGuideLine/stockid/{stock}/ctrl/2020/displaytype/4.phtml"
            r = requests.get(url)
            soup = BeautifulSoup(r.text, "lxml")
            if not y_list:
                year_context = soup.find(attrs={"id": "con02-1"}).find("table").find_all("a")
                year_list = [item.text for item in year_context]
            else:
                year_list = y_list
            out_df = pd.DataFrame()
            for year_item in tqdm(year_list):
                url = f"https://money.finance.sina.com.cn/corp/go.php/vFD_FinancialGuideLine/stockid/{stock}/ctrl/{year_item}/displaytype/4.phtml"
                # todo 爬数据一直超时 nodename nor servname provided, or not known
                r = requests.get(url, timeout=10)
                temp_df = pd.read_html(r.text)[12].iloc[:, :-1]
                if temp_df.shape[0] <= 1:
                    continue
                temp_df.columns = temp_df.iloc[0, :]
                temp_df = temp_df.iloc[1:, :]
                big_df = pd.DataFrame()
                for i in range(len(indicator_list)):
                    if i == len(indicator_list) - 1:
                        inner_df = temp_df[
                                   temp_df.loc[temp_df.iloc[:, 0].str.find(indicator_list[i]) == 0, :].index[0]:].T
                    else:
                        inner_df = temp_df[temp_df.loc[temp_df.iloc[:, 0].str.find(indicator_list[i]) == 0, :].index[0]:
                                           temp_df.loc[temp_df.iloc[:, 0].str.find(indicator_list[i + 1]) == 0,
                                           :].index[0] - 1].T
                    inner_df = inner_df.reset_index(drop=True)
                    big_df = pd.concat([big_df, inner_df], axis=1)
                big_df.columns = big_df.iloc[0, :].tolist()
                big_df = big_df.iloc[1:, :]
                big_df.index = temp_df.columns.tolist()[1:]
                out_df = out_df.append(big_df)
            out_df.dropna(inplace=True)
            return out_df

        for code in stock_all_df['code'].values:
            print("code:%s 获取分季度财务指标" % code)
            _id = "%s-%s-%s" % (code, indicator_list[0], sink_db_date)
            if code[0] not in ["0", "6"] or code[:3] == "688":
                # 科创板买不了，先不看！
                continue
            elif mongo_db.exists_one("collect_task_info", {"_id": _id}):
                print("task %s done" % _id)
            else:
                try:
                    result = stock_financial_analysis_indicator_fn(str(code), indicator_list)
                    if result.shape[0] == 0:
                        return
                    feature_dict = {"_id": code}
                    for d in result.index.tolist():
                        df = result[result.index == d]
                        # 2021Q1季度-净利润
                        d_token = d.split("-")
                        yyyy = d_token[0]
                        mm = int(d_token[1])
                        quarter = month_to_quarterly(mm)
                        prefix = "%sQ%s季度" % (yyyy, quarter)
                        df = df.rename(
                            columns={col: "%s-%s" % (prefix, col) for col in self.financial_indicator_columns})
                        # print(df)
                        # 稀疏列较多，考虑json或者mongo存储
                        dd = dict(zip(df.columns, df.head(1).values[0]))
                        dd = {k: v for k, v in dd.items() if
                              v not in ["每股指标", "盈利能力", "成长能力", "营运能力", "偿债及资本结构", "现金流量", "其他指标", "--"]}
                        feature_dict.update(dd)
                    # mongo_db.add("stock", feature_dict)
                    mongo_db.upsert("stock", {"_id": code}, feature_dict)
                    mongo_db.add("collect_task_info", {"_id": _id, "done": 1})
                except Exception as e:
                    sleep(1)
                    traceback.print_exc()
                    print("error stock_financial_analysis_indicator msg:%s" % repr(e))
                    continue

    def stock_financial_analysis_indicator_v3(self, date, indicator_list, y_list, mongo_db: MongoDB, stock_all_df):
        """财务指标(年度)
        目标地址:https://money.finance.sina.com.cn/corp/go.php/vFD_FinancialGuideLine/stockid/600004/ctrl/2019/displaytype/4.phtml
        """

        def stock_financial_analysis_indicator_fn(stock: str, indicator_list) -> pd.DataFrame:
            """
            新浪财经-财务分析-财务指标
            https://money.finance.sina.com.cn/corp/go.php/vFD_FinancialGuideLine/stockid/600004/ctrl/2019/displaytype/4.phtml
            :param stock: 股票代码
            :type stock: str
            :return: 新浪财经-财务分析-财务指标
            :rtype: pandas.DataFrame
            """
            url = f"https://money.finance.sina.com.cn/corp/go.php/vFD_FinancialGuideLine/stockid/{stock}/ctrl/2020/displaytype/4.phtml"
            r = requests.get(url)
            soup = BeautifulSoup(r.text, "lxml")
            if not y_list:
                year_context = soup.find(attrs={"id": "con02-1"}).find("table").find_all("a")
                year_list = [item.text for item in year_context]
            else:
                year_list = y_list
            out_df = pd.DataFrame()
            for year_item in tqdm(year_list):
                url = f"https://money.finance.sina.com.cn/corp/go.php/vFD_FinancialGuideLine/stockid/{stock}/ctrl/{year_item}/displaytype/4.phtml"
                # todo 爬数据一直超时 nodename nor servname provided, or not known
                r = requests.get(url, timeout=10)
                # fixme 拒绝访问,停止异常访问一段时间后(5~60分钟)会自动解封, 请耐心等待.
                temp_df = pd.read_html(r.text)[12].iloc[:, :-1]
                if temp_df.shape[0] <= 1:
                    continue
                temp_df.columns = temp_df.iloc[0, :]
                temp_df = temp_df.iloc[1:, :]
                big_df = pd.DataFrame()
                for i in range(len(indicator_list)):
                    if i == len(indicator_list) - 1:
                        inner_df = temp_df[
                                   temp_df.loc[temp_df.iloc[:, 0].str.find(indicator_list[i]) == 0, :].index[0]:].T
                    else:
                        inner_df = temp_df[temp_df.loc[temp_df.iloc[:, 0].str.find(indicator_list[i]) == 0, :].index[0]:
                                           temp_df.loc[temp_df.iloc[:, 0].str.find(indicator_list[i + 1]) == 0,
                                           :].index[0] - 1].T
                    inner_df = inner_df.reset_index(drop=True)
                    big_df = pd.concat([big_df, inner_df], axis=1)
                big_df.columns = big_df.iloc[0, :].tolist()
                big_df = big_df.iloc[1:, :]
                big_df.index = temp_df.columns.tolist()[1:]
                out_df = out_df.append(big_df)
            out_df.dropna(inplace=True)
            return out_df

        for code in stock_all_df['code'].values:
            print("code:%s 获取%s年度财务指标" % (code, y_list[0]))
            _id = "%s-%s-%s" % (code, indicator_list[0], date)
            if mongo_db.exists_one("collect_task_info", {"_id": _id}):
                print("task %s done" % _id)
            elif code[0] not in ["0", "6"]:
                # 科创板买不了，先不看！
                continue
            else:
                try:
                    result = stock_financial_analysis_indicator_fn(str(code), indicator_list)
                    if result.shape[0] == 0:
                        continue
                    feature_dict = {"_id": code}
                    year = y_list[0]
                    df = result[result.index == "%s-12-31" % year]
                    df = df.rename(columns={col: "%s年度-%s" % (year, col) for col in self.financial_indicator_columns})
                    if df.shape[0] == 0:
                        print("error code %s %s年度-财务指标获取失败!!!")
                        continue
                    dd = dict(zip(df.columns, df.head(1).values[0]))
                    dd = {k: double(v) for k, v in dd.items() if
                          v not in ["每股指标", "盈利能力", "成长能力", "营运能力", "偿债及资本结构", "现金流量", "其他指标", "--"]}
                    feature_dict.update(dd)
                    # mongo_db.add("stock", feature_dict)
                    mongo_db.upsert("stock", {"_id": code}, feature_dict)
                    mongo_db.add("collect_task_info", {"_id": _id, "done": 1})
                except Exception as e:
                    sleep(1)
                    traceback.print_exc()
                    print("error stock_financial_analysis_indicator msg:%s" % repr(e))
                    continue


if __name__ == "__main__":
    obj = StockBase()
    if len(sys.argv) > 1:
        date_s = sys.argv[1].strip()
        obj.date_s = date_s

    if not os.path.exists(ALL_STOCK_A_FILE):
        obj.download_all_stock()
    stock_all_df = pd.read_csv(ALL_STOCK_A_FILE, encoding="utf-8", dtype={"code": str})

    # 获取股票基本数据(所属行业，上市时间等),年更
    date_10 = """{YYYY}-12-31""".format(YYYY=last_year())
    date_8 = """{YYYY}1231""".format(YYYY=last_year())
    _id = "基本数据_%s" % date_8
    if mongo_db.exists_one("collect_task_info", {"_id": _id}):
        print("task %s done" % _id)
    else:
        obj.stock_basic()
        mongo_db.add("collect_task_info", {"_id": _id, "done": 1})

    # 获取历史十年来的年度财报
    for i in range(10, 0, -1):
        y = last_n_year(i, mode="str", format="YYYY")
        date_s = """{YYYY}1231""".format(YYYY=y)
        _id = "历史十年财报_%s" % date_s
        if mongo_db.exists_one("collect_task_info", {"_id": _id}):
            print("task %s done" % _id)
        else:
            print("starting download {date} 年报".format(date=date_s))
            obj.get_profit_statement_v2(date_s, stock_all_df)
            obj.get_cash_flow_statement_v2(date_s, stock_all_df)
            obj.get_assets_and_liabilities_statement_v2(date_s, stock_all_df)
            mongo_db.add("collect_task_info", {"_id": _id, "done": 1})

    # 获取A股历史十年财务指标
    for i in range(10, 0, -1):
        y = last_n_year(i, mode="str", format="YYYY")
        date_s = """{YYYY}1231""".format(YYYY=y)
        _id = "历史十年财务指标_%s" % date_s
        if mongo_db.exists_one("collect_task_info", {"_id": _id}):
            print("task %s done" % _id)
        else:
            print("starting download {date} 年度财务指标".format(date=date_s))
            obj.stock_financial_analysis_indicator_v3(date_s, ["每股指标"], [y], mongo_db, stock_all_df)
            mongo_db.add("collect_task_info", {"_id": _id, "done": 1})

    # 商誉数据,年更
    _id = "最新个股商誉数据_%s" % date_10
    if mongo_db.exists_one("collect_task_info", {"_id": _id}):
        print("task %s done" % _id)
    else:
        obj.get_goodwill_data_v2(date_10)
        mongo_db.add("collect_task_info", {"_id": _id, "done": 1})

    # 退市警告数据,年更
    _id = f"退市警告数据_{date_10}"
    if mongo_db.exists_one("collect_task_info", {"_id": _id}):
        # print("task %s done" % _id)
        pass
    else:
        obj.update_st_data(stock_all_df, date_10)
        mongo_db.add("collect_task_info", {"_id": _id, "done": 1})

    # 获取最新季度财报
    # obj.get_profit_statement_v2(quarterly_report_date(), stock_all_df)
    # obj.get_cash_flow_statement_v2(quarterly_report_date(), stock_all_df)
    # obj.get_assets_and_liabilities_statement_v2(quarterly_report_date(), stock_all_df)

    # 获取A股最新季度财务指标
    obj.stock_financial_analysis_indicator_v2(quarterly_report_date(), ["每股指标"], [now_year()], mongo_db, stock_all_df)

    # 获取A股每日估值指标
    obj.get_indicator_data_v2(obj.date_s, obj.date_s, "d", "3", mongo_db, stock_all_df)

    # k线数据
    obj.write_history_k_data(stock_all_df, obj.date_s, None, sync_update_month_k=True)
    # obj.write_history_k_data(stock_all_df, now_date_fmt_8(), None, frequency="m")
    # obj.stock_financial_analysis_indicator_v2(now_date_fmt_8(), ["每股指标"], ["2020"], mongo_db,
    #                                           stock_all_df)

    print("数据采集完成")
